var DoubanNGram = {
	docf: {},    // token->doc#
	tokenf: {},  // id -> Tokens, tokens=token:freq
	tfidf: {},   // id -> tfidf, tfidf=token:tfidf
	titles: {},  // id -> title
	doccount: 0,
	
	reset: function() {
		this.docf = {};
		this.tokenf = {};
		this.tfidf = {};
		this.titles = {};
		this.doccount = 0;
	},
	
	extractNGram: function(id, text) {
		var tokens = {};
		var content = text.replace(/(<([^>]+)>)/gi, "");
		var tokenCount = 0;
		for (var i = 1; i <= 4; ++i) {
			for (var j = 0; j <= content.length - i; ++j) {
				var token = content.substring(j, j + i);
				tokenCount ++;
				if (!tokens[token]) {
					tokens[token] = 0;
				}
				tokens[token]++;
			}
		}
		var output = id + " gram: ";
		for (var token in tokens) {
			if (!this.docf[token]) {
				this.docf[token] = 1;
			} else {
				this.docf[token] += 1;
			}
			tokens[token] = tokens[token] / tokenCount;
			output += token + "/" + tokens[token] + " ";
		}
		if (!this.tokenf[id]) {
			this.tokenf[id] = tokens;
			this.titles[id] = content;
			this.doccount++;
		}
//		Application.console.log(output);
	},
	
	getNGramSim: function(id) {
		var srctokens = this.getTfidf(id);
		if (srctokens == null)
			return null;
		
		var relitems = new Array();
		for (var item in this.tokenf) {
			if (item == id)
				continue;
			
			var desttokens = this.getTfidf(item);
			var cosine = 0;
			for (var token in srctokens) {
				if (desttokens[token]) {
					cosine += srctokens[token] * desttokens[token];
				}
			}
			cosine /= this.calculateRank(srctokens);
			cosine /= this.calculateRank(desttokens);
			if (cosine >= 0.075) {
				Application.console.log(this.titles[id] + " / " + this.titles[item] + ": " + cosine);
				var pair = {};
				pair.id = item;
				pair.cosine = cosine;
				relitems.push(pair);
			}	
		}
		relitems.sort(function(a, b) {
			return b.cosine - a.cosine;
		});
		var ids = new Array();
		for (var i = 0; i < relitems.length; ++i) {
			ids.push(relitems[i].id);
		}
		return ids;
	},
	
	calculateRank: function(tokens) {
		var rank = 0;
		for (var token in tokens) {
			rank += tokens[token] * tokens[token];
		}
		return Math.sqrt(rank);
	},
	
	getTfidf: function(id) {
		var srctokens = this.tfidf[id];
		if (!srctokens) {
			var srctokens = this.tokenf[id];
			if (!srctokens)
				return null;
			
			var output = {};
			for (var token in srctokens) {
				var tfidf = 0;
				if (this.docf[token]) {
					tfidf = srctokens[token] * Math.log(this.doccount / this.docf[token]);
				} else {
					tfidf = srctokens[token];
				}
				output[token] = tfidf;
			}
			this.tfidf[id] = output;
			srctokens = output;
		}
		return srctokens;
	}
}
